
import nbconvert
import nbformat

with open('NLP_reviews_summary.ipynb') as nb_file:
    nb_contents = nb_file.read()

# Convert using the ordinary exporter
notebook = nbformat.reads(nb_contents, as_version=4)
exporter = nbconvert.HTMLExporter()
body, res = exporter.from_notebook_node(notebook)


# Create a dict mapping all image attachments to their base64 representations
images = {}
for cell in notebook['cells']:
    if 'attachments' in cell:
        attachments = cell['attachments']
        for filename, attachment in attachments.items():
            for mime, base64 in attachment.items():
                images[f'attachment:{filename}'] = f'data:{mime};base64,{base64}'

# Fix up the HTML and write it to disk
for src, base64 in images.items():
    body = body.replace(f'src="{src}"', f'src="{base64}"')
with open('NLP_reviews_summary.html', 'w') as output_file:
    output_file.write(body)


import nltk
import sys
import json
from matplotlib import pyplot as plt
import time
import pandas as pd
import numpy as np
import string
import fasttext
import contractions
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords, wordnet
from nltk.stem import WordNetLemmatizer
from nltk import ne_chunk, pos_tag
from nltk.tree import Tree
import tensorflow as tf
from transformers import DistilBertTokenizerFast
from transformers import TFDistilBertForSequenceClassification
from transformers import DistilBertModel, DistilBertConfig
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from sklearn.utils import shuffle
import pickle
from os.path import exists


# import urllib
# todownload = 'http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_Apps_for_Android_5.json.gz'
# urllib.urlretrieve("todownload", "reviews_Apps_for_Android_5.json.gz")


amzr = []
for line in open('./data/reviews_Apps_for_Android_5.json', 'r'):
    amzr.append(json.loads(line))


columns = set([i for r in amzr for i in r.keys()])
# Create dictionarry -- this will be used to create our DataFrame
amzr_df = {c:[] for c in columns}


for row in amzr: # loop through data and append to dict
    for key in columns:
        amzr_df[key].append(row.get(key))
amzr_df = pd.DataFrame(amzr_df)


amzr_clean = amzr_df[~amzr_df['reviewText'].isnull()]
amzr_clean.drop_duplicates(subset=['reviewerID', 'asin', 'unixReviewTime'],inplace=True)
amzr_clean = amzr_clean.reset_index()


import plotly.express as px
toplot=amzr_clean[amzr_clean['overall'] != 'None']
toplot['overall'] = toplot['overall'].apply(lambda x: str(int(x)))
fig = px.histogram(toplot.astype(str), 
                   x = 'overall',
                  category_orders=dict(overall=["1", "2", "3","4",'5']),
                  color_discrete_sequence=['#ba7375','#ba7375','#ba7375','#ba7375','#ba7375'])
fig.update_layout(
    autosize=False,
    width=600,
    height=400,
)
fig.show()


toplot


amzr_clean['helpful_num'] = amzr_clean['helpful'].apply(lambda x: x[0])
amzr_clean['helpful_den'] = amzr_clean['helpful'].apply(lambda x: x[1])
amzr_clean['helpful_pct'] = np.where(amzr_clean['helpful_den'] > 0,
                                  amzr_clean['helpful_num'] / amzr_clean['helpful_den'], -1)
amzr_clean['helpfulness'] = pd.cut(x=amzr_clean['helpful_pct'], bins=[-1, 0, 0.2, 0.4, 0.6, 0.8, 1.0],
                                         labels=['None', '1', '2', '3', '4', '5'], include_lowest=True)
amzr_clean['sentiment'] = pd.cut(x=amzr_clean['overall'], bins=[-1, 1.5, 3.5, 6],
                                         labels=[0,1,2], include_lowest=True)
amzr_clean = amzr_clean.drop(columns=['helpful_num','helpful_pct'])


plt.figure(figsize=(6,4))
h=plt.hist(amzr_clean['reviewText'].apply(len),1000,color='#ba7375');
p=plt.xlim(0,1000);
p=plt.xlabel('Word Count')
p=plt.ylabel('Number of reviews')


excluded_pct = sum(amzr_clean['reviewText'].apply(len) > 256)/len(amzr_clean['reviewText'].apply(len))
print('We removed {:0.0f} % of the reviews'.format(excluded_pct*100))
amzr_clean = amzr_clean[amzr_clean['reviewText'].apply(len) <= 256]

We removed 21 % of the reviews


display(amzr_clean)


def process_text(df_in, cols=['reviewText','summary']):
    df =  df_in.copy()
    # Columns to drop (intermediate processed strings) 
    cols_ext_todrop = ['nocontract','nocontract_str','tokenized','lower','no_punc','pos_tags','wordnet_pos']
    stop_words = set(stopwords.words('english')) - {'not','no','nor',"aren't","isn't"}
    # Remove contractions
    for c in cols:
        df['nocontract'] = df[c].apply(lambda x: [contractions.fix(word) for word in x.split()])
        df['nocontract_str'] = [' '.join(map(str, l)) for l in df['nocontract']]
        # Tokenize
        df['tokenized'] = df['nocontract_str'].apply(word_tokenize)
        # make lower-case
        df['lower'] = df['tokenized'].apply(lambda x: [word.lower() for word in x])
        # remove punctuation
        punc = string.punctuation
        df['no_punc'] = df['lower'].apply(lambda x: [word for word in x if word not in punc])
#         # remove stop-words
#         df['stopwords_removed'] = df['no_punc'].apply(lambda x: [word for word in x if word not in stop_words])
        # tag words
        df['pos_tags'] = df['no_punc'].apply(nltk.tag.pos_tag)
        # lemmatize
        def get_wordnet_pos(tag):
            if tag.startswith('J'):
                return wordnet.ADJ
            elif tag.startswith('V'):
                return wordnet.VERB
            elif tag.startswith('N'):
                return wordnet.NOUN
            elif tag.startswith('R'):
                return wordnet.ADV
            else:
                return wordnet.NOUN
        df['wordnet_pos'] = df['pos_tags'].apply(lambda x: [(word, get_wordnet_pos(pos_tag)) for (word, pos_tag) in x])
        wnl = WordNetLemmatizer()
        df[c+'_processed'] = df['wordnet_pos'].apply(lambda x: " ".join([wnl.lemmatize(word, tag) for word, tag in x]))
        # drop intermediate columns
        df=df.drop(columns=cols_ext_todrop)
    return df


nfilt = 5 # number of ratings threshold
amzr_keep_helful = amzr_clean[(amzr_clean['helpfulness'].isin(['3','4','5'])) & (amzr_clean['helpful_den'].astype(int) >= nfilt)]


n_train=10000 # training sample
n_test=10000 # testing sample


amzr_clean_sample = process_text(amzr_clean.sample(n=n_train + n_test, replace=False, random_state=1))
amzr_test_sample = amzr_clean_sample.iloc[n_train:]
amzr_clean_sample = amzr_clean_sample.iloc[:n_train]


amzr_keep_helful[~amzr_keep_helful.isin(amzr_test_sample)].dropna() # remove rows from test sample
amzr_keep_helful_sample = process_text(amzr_keep_helful.sample(n=n_train + n_test, replace=False, random_state=1))
amzr_keep_helful_test_sample = amzr_keep_helful_sample.iloc[n_train:]
amzr_keep_helful_sample = amzr_keep_helful_sample.iloc[:n_train]


amzr_clean[~amzr_clean.isin(amzr_test_sample)].dropna() # remove rows from test sample

g = amzr_clean.groupby('sentiment')
amzr_clean_sample_balanced=shuffle(process_text(pd.DataFrame(g.apply(lambda x: x.sample(int((n_train + n_test)/3),replace=False))))).reset_index(drop=True)
amzr_test_sample_balanced = amzr_clean_sample_balanced.iloc[n_train:]
amzr_clean_sample_balanced = amzr_clean_sample_balanced.iloc[:n_train]


amzr_keep_helful[~amzr_keep_helful.isin(amzr_test_sample)].dropna() # remove rows from test sample

g = amzr_keep_helful.groupby('sentiment')
amzr_keep_helful_sample_balanced=shuffle(process_text(pd.DataFrame(g.apply(lambda x: x.sample(int((n_train + n_test)/3),replace=False))))).reset_index(drop=True)
amzr_keep_helful_test_sample_balanced = amzr_keep_helful_sample_balanced.iloc[n_train:]
amzr_keep_helful_sample_balanced = amzr_keep_helful_sample_balanced.iloc[:n_train]


amzr_clean_sample_balanced


model_results_path = './out/model_results.pkl'
if exists(model_results_path):
    with open('./out/model_results.pkl', 'rb') as f:  
        model_results = pickle.load(f)  
else:
    model_results = {}


def set_up_model(df,features ='reviewText_processed',labels = 'sentiment',max_length=256):
    reviews = df[features].values.tolist()
    labels = df[labels].tolist()
    training_sentences, validation_sentences, training_labels, validation_labels = train_test_split(reviews, labels, test_size=.2)
    tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
    train_encodings = tokenizer(training_sentences,
                                truncation=True,
                                padding=True,
                                max_length=max_length)
    val_encodings = tokenizer(validation_sentences,
                                truncation=True,
                                padding=True,
                                max_length=max_length)
    train_dataset = tf.data.Dataset.from_tensor_slices((
                                dict(train_encodings),
                                training_labels
                                ))
    val_dataset = tf.data.Dataset.from_tensor_slices((
                                dict(val_encodings),
                                validation_labels
                                ))
    callback = tf.keras.callbacks.EarlyStopping(monitor='loss',patience = 2)
    model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased',num_labels=3)
    model.config.dropout = 0.3
    optimizer = tf.keras.optimizers.Adam(learning_rate=2.5e-5, epsilon=1e-08)
    model.compile(optimizer=optimizer, loss=model.compute_loss, metrics=['accuracy'])
    return model, train_dataset, val_dataset


def compute_test_accuracy(df,model, features = 'reviewText_processed',labels='sentiment',max_length=256):
    tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
    test_sentences = df[features].values.tolist()
    test_labels = df[labels].tolist()
    tf_batch = [tokenizer(i, padding=True, truncation=True, max_length=max_length, return_tensors='tf') for i in test_sentences]
    tf_outputs = [model(i) for i in tf_batch]
    tf_predictions = [tf.nn.softmax(i[0], axis=-1) for i in tf_outputs]
    pred_test_labels = [tf.argmax(i, axis=1).numpy()[0] for i in tf_predictions]
    metric = tf.keras.metrics.Accuracy()
    metric.update_state(test_labels,pred_test_labels)
    return metric.result().numpy(),test_labels,pred_test_labels


# train_cases = {'balanced':amzr_clean_sample_balanced,'unbalanced':amzr_clean_sample,'helpful_balanced':amzr_keep_helful_sample_balanced}
train_cases = {'balanced':amzr_clean_sample_balanced,'unbalanced':amzr_clean_sample,'helpful_unbalanced':amzr_keep_helful_sample}
# feature_cases = ['reviewText','reviewText_processed','summary','summary_processed']
#train_cases = {'helpful_unbalanced':amzr_keep_helful_sample,'unbalanced':amzr_clean_sample}
feature_cases = ['reviewText','summary']
max_length = 256
batch_size = [4,8]
mode = 'prod4'
for b in batch_size:
    for name,t in train_cases.items():
        for feat in feature_cases:
            print('running '+'run_'+name+"_"+feat+"_"+str(b)+"_"+mode)
            if feat in ['summary','summary_processed']:
                max_length = 64
            model, train_dataset, val_dataset= set_up_model(t,max_length=max_length)
            history = model.fit(train_dataset.shuffle(100).batch(b),
                  epochs=2,
                  batch_size=b,
                  validation_data=val_dataset.shuffle(100).batch(b))
            acc,test_labels,pred_test_labels = compute_test_accuracy(amzr_test_sample,model,max_length=max_length)
            model_results['run_'+name+"_"+feat+"_"+"_"+str(b)+"_"+mode]  = {'parameters':{'max_length':max_length,'batch_size':batch_size,'Case':name},'test_acc':acc,'history_acc':history.history['accuracy'],'history_valacc':history.history['val_accuracy'],'test_labels':test_labels,"pred_test_labels":pred_test_labels}
            model.save_pretrained("./models/sentiment_run_"+name+"_"+feat+"_"+"_"+str(b)+"_"+mode)
            with open('./out/model_results.pkl', 'wb') as f:  
                pickle.dump(model_results, f)  
            cm=confusion_matrix(pred_test_labels, test_labels)
            cm = cm/np.transpose(np.tile(np.sum(cm,axis=1),(3,1)))
            disp = ConfusionMatrixDisplay(confusion_matrix=cm,
                               display_labels=[0,1,2])
            disp.plot(cmap="YlGn")
            plt.savefig('./img/conf_run_'+name+"_"+feat+"_"+"_"+str(b)+"_"+mode+'.png')
            plt.savefig('./img/conf_run_'+str(name)+"_"+str(feat)+"_"+"_"+str(b)+"_"+mode+'.svg')

running run_balanced_reviewText_4_prod4

2021-09-28 04:16:38.966275: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
2021-09-28 04:16:39.000309: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.000664: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: 
pciBusID: 0000:01:00.0 name: NVIDIA GeForce GTX 1660 computeCapability: 7.5
coreClock: 1.86GHz coreCount: 22 deviceMemorySize: 5.80GiB deviceMemoryBandwidth: 178.86GiB/s
2021-09-28 04:16:39.000713: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2021-09-28 04:16:39.000764: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2021-09-28 04:16:39.000799: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2021-09-28 04:16:39.000833: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2021-09-28 04:16:39.001956: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2021-09-28 04:16:39.001990: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2021-09-28 04:16:39.002015: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2021-09-28 04:16:39.002073: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.002360: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.002604: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0
2021-09-28 04:16:39.002953: I tensorflow/core/platform/cpu_feature_guard.cc:143] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA
2021-09-28 04:16:39.026080: I tensorflow/core/platform/profile_utils/cpu_utils.cc:102] CPU Frequency: 3999980000 Hz
2021-09-28 04:16:39.026412: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5567317666a0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-09-28 04:16:39.026430: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
2021-09-28 04:16:39.088852: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.089147: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x556732db3690 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2021-09-28 04:16:39.089159: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce GTX 1660, Compute Capability 7.5
2021-09-28 04:16:39.089313: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.089539: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: 
pciBusID: 0000:01:00.0 name: NVIDIA GeForce GTX 1660 computeCapability: 7.5
coreClock: 1.86GHz coreCount: 22 deviceMemorySize: 5.80GiB deviceMemoryBandwidth: 178.86GiB/s
2021-09-28 04:16:39.089575: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2021-09-28 04:16:39.089590: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2021-09-28 04:16:39.089602: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2021-09-28 04:16:39.089615: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2021-09-28 04:16:39.089628: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2021-09-28 04:16:39.089641: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2021-09-28 04:16:39.089653: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2021-09-28 04:16:39.089698: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.089951: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.090154: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0
2021-09-28 04:16:39.090180: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2021-09-28 04:16:39.856425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] Device interconnect StreamExecutor with strength 1 edge matrix:
2021-09-28 04:16:39.856449: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1108]      0 
2021-09-28 04:16:39.856454: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1121] 0:   N 
2021-09-28 04:16:39.856644: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.857055: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-09-28 04:16:39.857375: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1247] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 4262 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce GTX 1660, pci bus id: 0000:01:00.0, compute capability: 7.5)
2021-09-28 04:16:42.563320: W tensorflow/python/util/util.cc:329] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
2021-09-28 04:16:42.936291: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForSequenceClassification: ['activation_13', 'vocab_projector', 'vocab_transform', 'vocab_layer_norm']
- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier', 'pre_classifier', 'dropout_19']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Epoch 1/2
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.Socket(zmq.PUSH) at 0x7fbf643c9160>> and will run it as-is.
Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.
Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method
To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert
WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.Socket(zmq.PUSH) at 0x7fbf643c9160>> and will run it as-is.
Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.
Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method
To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - ETA: 0s - loss: 0.7046 - accuracy: 0.6858WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - 204s 102ms/step - loss: 0.7046 - accuracy: 0.6858 - val_loss: 0.6328 - val_accuracy: 0.7375
Epoch 2/2
2000/2000 [==============================] - 204s 102ms/step - loss: 0.5047 - accuracy: 0.7889 - val_loss: 0.6758 - val_accuracy: 0.7155
running run_balanced_summary_4_prod4

Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForSequenceClassification: ['activation_13', 'vocab_projector', 'vocab_transform', 'vocab_layer_norm']
- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier', 'dropout_39', 'pre_classifier']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Epoch 1/2
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - ETA: 0s - loss: 0.7159 - accuracy: 0.6760WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - 138s 69ms/step - loss: 0.7159 - accuracy: 0.6760 - val_loss: 0.6396 - val_accuracy: 0.7175
Epoch 2/2
2000/2000 [==============================] - 137s 69ms/step - loss: 0.5293 - accuracy: 0.7744 - val_loss: 0.6493 - val_accuracy: 0.7255
running run_unbalanced_reviewText_4_prod4

Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForSequenceClassification: ['activation_13', 'vocab_projector', 'vocab_transform', 'vocab_layer_norm']
- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier', 'pre_classifier', 'dropout_59']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Epoch 1/2
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - ETA: 0s - loss: 0.4938 - accuracy: 0.8043WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - 138s 69ms/step - loss: 0.4938 - accuracy: 0.8043 - val_loss: 0.4419 - val_accuracy: 0.8395
Epoch 2/2
2000/2000 [==============================] - 137s 69ms/step - loss: 0.3461 - accuracy: 0.8648 - val_loss: 0.4778 - val_accuracy: 0.8355
running run_unbalanced_summary_4_prod4

Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForSequenceClassification: ['activation_13', 'vocab_projector', 'vocab_transform', 'vocab_layer_norm']
- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier', 'dropout_79', 'pre_classifier']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Epoch 1/2
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - ETA: 0s - loss: 0.4774 - accuracy: 0.8050WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
2000/2000 [==============================] - 139s 69ms/step - loss: 0.4774 - accuracy: 0.8050 - val_loss: 0.4500 - val_accuracy: 0.8230
Epoch 2/2
2000/2000 [==============================] - 137s 68ms/step - loss: 0.3225 - accuracy: 0.8700 - val_loss: 0.5301 - val_accuracy: 0.7940
running run_helpful_unbalanced_reviewText_4_prod4

Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForSequenceClassification: ['activation_13', 'vocab_projector', 'vocab_transform', 'vocab_layer_norm']
- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier', 'pre_classifier', 'dropout_99']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.


# for i,k in enumerate(model_results):
#     if len(np.array(model_results[k]['pred_test_labels'])) == 4000:
#         continue
#     if i == 0: 
#         preds = np.array(model_results[k]['pred_test_labels'])
#         test = np.array(model_results[k]['pred_test_labels'])
#     if len(preds.shape)==1:
#         preds = np.concatenate((preds[:,np.newaxis],np.array(model_results[k]['pred_test_labels'])[:,np.newaxis]),axis=1)
#         test = np.concatenate((test[:,np.newaxis],np.array(model_results[k]['test_labels'])[:,np.newaxis]),axis=1)
#     else:
#         preds = np.concatenate((preds,np.array(model_results[k]['pred_test_labels'])[:,np.newaxis]),axis=1)
#         test = np.concatenate((test,np.array(model_results[k]['test_labels'])[:,np.newaxis]),axis=1)

	index	reviewerID	unixReviewTime	reviewerName	summary	reviewTime	overall	reviewText	helpful	asin
0	0	A1N4O8VOJZTDVB	1383350400	Annette Yancey	Really cute	11 2, 2013	3	Loves the song, so he really couldn't wait to ...	[1, 1]	B004A9SDD8
1	1	A2HQWU6HUKIEC7	1323043200	Audiobook lover "Kathy"	2-year-old loves it	12 5, 2011	5	Oh, how my little grandson loves this app. He'...	[0, 0]	B004A9SDD8
2	2	A1SXASF6GYG96I	1337558400	Barbara Gibbs	Fun game	05 21, 2012	5	I found this at a perfect time since my daught...	[0, 0]	B004A9SDD8
3	3	A2B54P9ZDYH167	1354752000	Brooke Greenstreet "Babylove"	We love our Monkeys!	12 6, 2012	5	My 1 year old goes back to this game over and ...	[3, 4]	B004A9SDD8
4	4	AFOFZDTX5UC6D	1391212800	C. Galindo	This is my granddaughters favorite app on my K...	02 1, 2014	5	There are three different versions of the song...	[1, 1]	B004A9SDD8
...	...	...	...	...	...	...	...	...	...	...
752932	752932	AZJ11YS0E52AI	1405814400	K. Perna	Five Stars	07 20, 2014	5	I love it!!!!!!!!!! really keeps your attenti...	[0, 0]	B00LUEMK44
752933	752933	A2550XGZEFDH2Y	1405900800	Melanie G. Nihart "Grammy"	... are so many free ones that are so much bet...	07 21, 2014	3	Okay but there are so many free ones that are ...	[0, 0]	B00LUEMK44
752934	752934	A1KNDB16TG5QXD	1405900800	P. O'Reilly	Enjoyable	07 21, 2014	4	Another great jewels game that just keeps you ...	[0, 0]	B00LUEMK44
752935	752935	A1IHFHA5LI9SGI	1405814400	redhatflusher	entertaining	07 20, 2014	5	I find this the best jewels star ever. There s...	[0, 0]	B00LUEMK44
752936	752936	A3GVFRVK9RYGTV	1405728000	Smiley	This game was fun to play but was not as good ...	07 19, 2014	4	This game was fun to play but was not as good ...	[0, 0]	B00LUEMK44

	index	reviewerID	unixReviewTime	reviewerName	summary	reviewTime	overall	reviewText	helpful	asin	helpful_den	helpfulness	sentiment
0	0	A1N4O8VOJZTDVB	1383350400	Annette Yancey	Really cute	11 2, 2013	3.0	Loves the song, so he really couldn't wait to ...	[1, 1]	B004A9SDD8	1	5	1
1	1	A2HQWU6HUKIEC7	1323043200	Audiobook lover "Kathy"	2-year-old loves it	12 5, 2011	5.0	Oh, how my little grandson loves this app. He'...	[0, 0]	B004A9SDD8	0	None	2
3	3	A2B54P9ZDYH167	1354752000	Brooke Greenstreet "Babylove"	We love our Monkeys!	12 6, 2012	5.0	My 1 year old goes back to this game over and ...	[3, 4]	B004A9SDD8	4	4	2
5	5	A331GYAT4ESYI3	1354665600	Felicia	so cute	12 5, 2012	5.0	THis is just so cute and a great app for littl...	[3, 3]	B004A9SDD8	3	5	2
7	7	A3699WHISXX94Z	1389484800	Janie Leonard	Five Little Monkeys	01 12, 2014	5.0	This app is wild and crazy. Little ones love ...	[0, 0]	B004A9SDD8	0	None	2
...	...	...	...	...	...	...	...	...	...	...	...	...	...
752932	752932	AZJ11YS0E52AI	1405814400	K. Perna	Five Stars	07 20, 2014	5.0	I love it!!!!!!!!!! really keeps your attenti...	[0, 0]	B00LUEMK44	0	None	2
752933	752933	A2550XGZEFDH2Y	1405900800	Melanie G. Nihart "Grammy"	... are so many free ones that are so much bet...	07 21, 2014	3.0	Okay but there are so many free ones that are ...	[0, 0]	B00LUEMK44	0	None	1
752934	752934	A1KNDB16TG5QXD	1405900800	P. O'Reilly	Enjoyable	07 21, 2014	4.0	Another great jewels game that just keeps you ...	[0, 0]	B00LUEMK44	0	None	2
752935	752935	A1IHFHA5LI9SGI	1405814400	redhatflusher	entertaining	07 20, 2014	5.0	I find this the best jewels star ever. There s...	[0, 0]	B00LUEMK44	0	None	2
752936	752936	A3GVFRVK9RYGTV	1405728000	Smiley	This game was fun to play but was not as good ...	07 19, 2014	4.0	This game was fun to play but was not as good ...	[0, 0]	B00LUEMK44	0	None	2

	index	reviewerID	unixReviewTime	reviewerName	summary	reviewTime	overall	reviewText	helpful	asin	helpful_den	helpfulness	sentiment	reviewText_processed	summary_processed
0	99856	A11LDVCPN8F60Y	1341878400	Christian D. Costello "urban cowboy"	doesn't even load	07 10, 2012	1.0	I currently have the highest end android devic...	[1, 16]	B005DTZAZS	16	1	0	i currently have the high end android device o...	do not even load
1	469710	A1EXW1CPDBBPQJ	1355702400	Augie (A.K.A c9g7k2)	The nether	12 17, 2012	1.0	They reloaded the nether from minecraft and ca...	[480, 549]	B00AEC85Z6	549	5	0	they reload the nether from minecraft and call...	the nether
2	703027	A27FBOHUWC1GQS	1387756800	if you are like me you would love this game!	adorable...	12 23, 2013	4.0	If you love cute stuff than this game is for y...	[28, 29]	B00GUY712U	29	5	2	if you love cute stuff than this game be for y...	adorable ...
3	280691	A1ZGITFDYBP37C	1355184000	Awesome Man	Loved It	12 11, 2012	5.0	Awesome and simpleThat describes a lotI RECOMM...	[0, 0]	B007ZGO7EM	0	None	2	awesome and simplethat describe a loti recomme...	love it
4	321901	A3SV4GOS0B34ZK	1364601600	Lynn	Did not work	03 30, 2013	1.0	I tried to download this app as well and like ...	[0, 0]	B008AZGWYK	0	None	0	i try to download this app as well and like th...	do not work
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
9995	674939	A3PVE2P1WD6FN8	1391731200	Avid Reader Girl	nice little time waster	02 7, 2014	3.0	Got it to kill some time before an appointment...	[0, 0]	B00FELZD70	0	None	1	get it to kill some time before an appointment...	nice little time waster
9996	554654	A2BQI081NL5FNX	1379808000	Amazon Customer	Good game	09 22, 2013	4.0	May be too hard for young children but for 10 ...	[3, 5]	B00C1C2LMK	5	3	2	may be too hard for young child but for 10 and...	good game
9997	549750	AYC1XIHJOWQBO	1391558400	ochocki	Great for kids!	02 5, 2014	5.0	It is a good game for my 7 year old. he loves ...	[0, 0]	B00BWBHIUG	0	None	2	it be a good game for my 7 year old he love th...	great for kid
9998	105944	A24CEFMYBRZ2WI	1387670400	Jaxon X. Boozer	Give me the 100 pin for free...NOW!	12 22, 2013	2.0	You guys aren't cool. You won't let me play th...	[1, 2]	B005IDXIL2	2	3	1	you guy be not cool you will not let me play t...	give me the 100 pin for free ... now
9999	734794	A30V9M9DZW8SFU	1398988800	Gypsy	cute	05 2, 2014	1.0	but cute does not cut it, the sample alarms ar...	[0, 1]	B00IIOMDT6	1	None	0	but cute do not cut it the sample alarm be lou...	cute

Case Study -- NLP Amazon Reviews (code)¶

Version 0.1¶

Simon Yang¶

Import libraries and dependant data¶

Download Amazon review data (optional)¶

Read data¶

Repackage data into pandas dataframe¶

First we retrieve all possible unique user key to define our table's columns¶

Let's fill our table with data¶

Clean-up data:¶

Retreive helpfullness and bin into categories:¶

Filter for reviews with less than 256 words¶

Text processing¶

Inspect Helpfulness¶

Using helpulness to filter the training data¶

Sample data for training and testing (balanced and imbalanced)¶

Display example training data (Case 1)¶

Train model¶

Check if we already have some results already¶

Function for setting up our model¶

Function for testing our model¶

Train multiple instances of the model with various features and parameters and evaluate performance¶